COCO Image Viewer

This notebook will allow you to view details about a COCO dataset and preview segmentations on annotated images. Learn more about it at: http://cocodataset.org/

In [1]:
import IPython
import os
import json
import random
import numpy as np
import requests
from io import BytesIO
import base64
from math import trunc
from PIL import Image as PILImage
from PIL import ImageDraw as PILImageDraw

Image Viewer Logic

In [2]:
# Load the dataset json
class CocoDataset():
    def __init__(self, annotation_path, image_dir):
        self.annotation_path = annotation_path
        self.image_dir = image_dir
        self.colors = colors = ['blue', 'purple', 'red', 'green', 'orange', 'salmon', 'pink', 'gold',
                                'orchid', 'slateblue', 'limegreen', 'seagreen', 'darkgreen', 'olive',
                               'teal', 'aquamarine', 'steelblue', 'powderblue', 'dodgerblue', 'navy',
                               'magenta', 'sienna', 'maroon']
        
        json_file = open(self.annotation_path)
        self.coco = json.load(json_file)
        json_file.close()
        
        self.process_info()
        self.process_licenses()
        self.process_categories()
        self.process_images()
        self.process_segmentations()
            
        
    def display_info(self):
        print('Dataset Info:')
        print('=============')
        for key, item in self.info.items():
            print('  {}: {}'.format(key, item))
        
        requirements = [['description', str],
                        ['url', str],
                        ['version', str],
                        ['year', int],
                        ['contributor', str],
                        ['date_created', str]]
        for req, req_type in requirements:
            if req not in self.info:
                print('ERROR: {} is missing'.format(req))
            elif type(self.info[req]) != req_type:
                print('ERROR: {} should be type {}'.format(req, str(req_type)))
        print('')

        
    def display_licenses(self):
        print('Licenses:')
        print('=========')
        
        requirements = [['id', int],
                        ['url', str],
                        ['name', str]]
        for license in self.licenses:
            for key, item in license.items():
                print('  {}: {}'.format(key, item))
            for req, req_type in requirements:
                if req not in license:
                    print('ERROR: {} is missing'.format(req))
                elif type(license[req]) != req_type:
                    print('ERROR: {} should be type {}'.format(req, str(req_type)))
            print('')
        print('')
        
    def display_categories(self):
        print('Categories:')
        print('=========')
        for sc_key, sc_val in self.super_categories.items():
            print('  super_category: {}'.format(sc_key))
            for cat_id in sc_val:
                print('    id {}: {}'.format(cat_id, self.categories[cat_id]['name']))
            print('')
    
    def display_image(self, image_id, show_polys=True, show_bbox=True, show_crowds=True, use_url=False):
        print('Image:')
        print('======')
        if image_id == 'random':
            image_id = random.choice(list(self.images.keys()))
        
        # Print the image info
        image = self.images[image_id]
        for key, val in image.items():
            print('  {}: {}'.format(key, val))
            
        # Open the image
        if use_url:
            image_path = image['coco_url']
            response = requests.get(image_path)
            image = PILImage.open(BytesIO(response.content))
            
        else:
            image_path = os.path.join(self.image_dir, image['file_name'])
            image = PILImage.open(image_path)
            
            buffer = BytesIO()
            image.save(buffer, format='PNG')
            buffer.seek(0)
            
            data_uri = base64.b64encode(buffer.read()).decode('ascii')
            image_path = "data:image/png;base64,{0}".format(data_uri)
            
        # Calculate the size and adjusted display size
        max_width = 600
        image_width, image_height = image.size
        adjusted_width = min(image_width, max_width)
        adjusted_ratio = adjusted_width / image_width
        adjusted_height = adjusted_ratio * image_height
        
        # Create list of polygons to be drawn
        polygons = {}
        bbox_polygons = {}
        rle_regions = {}
        poly_colors = {}
        print('  segmentations ({}):'.format(len(self.segmentations[image_id])))
        for i, segm in enumerate(self.segmentations[image_id]):
            polygons_list = []
            if segm['iscrowd'] != 0:
                # Gotta decode the RLE
                px = 0
                x, y = 0, 0
                rle_list = []
                for j, counts in enumerate(segm['segmentation']['counts']):
                    if j % 2 == 0:
                        # Empty pixels
                        px += counts
                    else:
                        # Need to draw on these pixels, since we are drawing in vector form,
                        # we need to draw horizontal lines on the image
                        x_start = trunc(trunc(px / image_height) * adjusted_ratio)
                        y_start = trunc(px % image_height * adjusted_ratio)
                        px += counts
                        x_end = trunc(trunc(px / image_height) * adjusted_ratio)
                        y_end = trunc(px % image_height * adjusted_ratio)
                        if x_end == x_start:
                            # This is only on one line
                            rle_list.append({'x': x_start, 'y': y_start, 'width': 1 , 'height': (y_end - y_start)})
                        if x_end > x_start:
                            # This spans more than one line
                            # Insert top line first
                            rle_list.append({'x': x_start, 'y': y_start, 'width': 1, 'height': (image_height - y_start)})
                            
                            # Insert middle lines if needed
                            lines_spanned = x_end - x_start + 1 # total number of lines spanned
                            full_lines_to_insert = lines_spanned - 2
                            if full_lines_to_insert > 0:
                                full_lines_to_insert = trunc(full_lines_to_insert * adjusted_ratio)
                                rle_list.append({'x': (x_start + 1), 'y': 0, 'width': full_lines_to_insert, 'height': image_height})
                                
                            # Insert bottom line
                            rle_list.append({'x': x_end, 'y': 0, 'width': 1, 'height': y_end})
                if len(rle_list) > 0:
                    rle_regions[segm['id']] = rle_list  
            else:
                # Add the polygon segmentation
                for segmentation_points in segm['segmentation']:
                    segmentation_points = np.multiply(segmentation_points, adjusted_ratio).astype(int)
                    polygons_list.append(str(segmentation_points).lstrip('[').rstrip(']'))
            polygons[segm['id']] = polygons_list
            if i < len(self.colors):
                poly_colors[segm['id']] = self.colors[i]
            else:
                poly_colors[segm['id']] = 'white'
            
            bbox = segm['bbox']
            bbox_points = [bbox[0], bbox[1], bbox[0] + bbox[2], bbox[1],
                           bbox[0] + bbox[2], bbox[1] + bbox[3], bbox[0], bbox[1] + bbox[3],
                           bbox[0], bbox[1]]
            bbox_points = np.multiply(bbox_points, adjusted_ratio).astype(int)
            bbox_polygons[segm['id']] = str(bbox_points).lstrip('[').rstrip(']')
            
            # Print details
            print('    {}:{}:{}'.format(segm['id'], poly_colors[segm['id']], self.categories[segm['category_id']]))
        
        
        
        # Draw segmentation polygons on image
        html  = '<div class="container" style="position:relative;">'
        html += '<img src="{}" style="position:relative;top:0px;left:0px;width:{}px;">'.format(image_path, adjusted_width)
        html += '<div class="svgclass"><svg width="{}" height="{}">'.format(adjusted_width, adjusted_height)
        
        if show_polys:
            for seg_id, points_list in polygons.items():
                fill_color = poly_colors[seg_id]
                stroke_color = poly_colors[seg_id]
                for points in points_list:
                    html += '<polygon points="{}" style="fill:{}; stroke:{}; stroke-width:1; fill-opacity:0.5" />'.format(points, fill_color, stroke_color)
        
        if show_crowds:
            for seg_id, rect_list in rle_regions.items():
                fill_color = poly_colors[seg_id]
                stroke_color = poly_colors[seg_id]
                for rect_def in rect_list:
                    x, y = rect_def['x'], rect_def['y']
                    w, h = rect_def['width'], rect_def['height']
                    html += '<rect x="{}" y="{}" width="{}" height="{}" style="fill:{}; stroke:{}; stroke-width:1; fill-opacity:0.5; stroke-opacity:0.5" />'.format(x, y, w, h, fill_color, stroke_color)
            
        if show_bbox:
            for seg_id, points in bbox_polygons.items():
                fill_color = poly_colors[seg_id]
                stroke_color = poly_colors[seg_id]
                html += '<polygon points="{}" style="fill:{}; stroke:{}; stroke-width:1; fill-opacity:0" />'.format(points, fill_color, stroke_color)
                
        html += '</svg></div>'
        html += '</div>'
        html += '<style>'
        html += '.svgclass { position:absolute; top:0px; left:0px;}'
        html += '</style>'
        return html
       
    def process_info(self):
        self.info = self.coco['info']
    
    def process_licenses(self):
        self.licenses = self.coco['licenses']
    
    def process_categories(self):
        self.categories = {}
        self.super_categories = {}
        for category in self.coco['categories']:
            cat_id = category['id']
            super_category = category['supercategory']
            
            # Add category to the categories dict
            if cat_id not in self.categories:
                self.categories[cat_id] = category
            else:
                print("ERROR: Skipping duplicate category id: {}".format(category))

            # Add category to super_categories dict
            if super_category not in self.super_categories:
                self.super_categories[super_category] = {cat_id} # Create a new set with the category id
            else:
                self.super_categories[super_category] |= {cat_id} # Add category id to the set
                
    def process_images(self):
        self.images = {}
        for image in self.coco['images']:
            image_id = image['id']
            if image_id in self.images:
                print("ERROR: Skipping duplicate image id: {}".format(image))
            else:
                self.images[image_id] = image
                
    def process_segmentations(self):
        self.segmentations = {}
        for segmentation in self.coco['annotations']:
            image_id = segmentation['image_id']
            if image_id not in self.segmentations:
                self.segmentations[image_id] = []
            self.segmentations[image_id].append(segmentation)

Explore Image Meta Data

In [3]:
annotation_path = 'IB-SEC.v2i.coco/_annotations.coco.test.json'
filtered_annotation_path = 'IB-SEC.v2i.coco/_annotations.coco.modified.test.json'
image_dir = 'IB-SEC.v2i.coco/test'

coco_dataset = CocoDataset(annotation_path, image_dir)
f_coco_dataset = CocoDataset(filtered_annotation_path, image_dir)
coco_dataset.display_info()
coco_dataset.display_licenses()
coco_dataset.display_categories()
Dataset Info:
=============
  year: 2022
  version: 2
  description: Exported from roboflow.ai
  contributor: 
  url: https://public.roboflow.ai/object-detection/undefined
  date_created: 2022-03-10T03:45:06+00:00
ERROR: year should be type <class 'int'>

Licenses:
=========
  id: 1
  url: https://creativecommons.org/licenses/by/4.0/
  name: CC BY 4.0


Categories:
=========
  super_category: none
    id 0: cube-doc

  super_category: cube-doc
    id 1: Issuance_date
    id 2: author
    id 3: bullet_enum
    id 4: contact_info
    id 5: data_information
    id 6: foot_note
    id 7: footer
    id 8: header
    id 9: heading
    id 10: note
    id 11: paragraph
    id 12: reference
    id 13: table

Display sample images - before annotation filtration

In [4]:
res = random.sample(range(0, 30), 7)
for img_id in res:
    html = coco_dataset.display_image(img_id, use_url=False)
    IPython.display.display(IPython.display.HTML(html))
Image:
======
  id: 10
  license: 1
  file_name: 34-90543_0007_jpg.rf.24a847064d049989b350f9aaa88576dc.jpg
  height: 2200
  width: 1700
  date_captured: 2022-03-10T03:45:06+00:00
  segmentations (5):
    60:blue:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
    61:purple:{'id': 9, 'name': 'heading', 'supercategory': 'cube-doc'}
    62:red:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
    63:green:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
    64:orange:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
Image:
======
  id: 4
  license: 1
  file_name: 34-90552_0008_jpg.rf.87776dd4a412222469ba63b707982945.jpg
  height: 2200
  width: 1700
  date_captured: 2022-03-10T03:45:06+00:00
  segmentations (9):
    22:blue:{'id': 9, 'name': 'heading', 'supercategory': 'cube-doc'}
    23:purple:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
    24:red:{'id': 9, 'name': 'heading', 'supercategory': 'cube-doc'}
    25:green:{'id': 3, 'name': 'bullet_enum', 'supercategory': 'cube-doc'}
    26:orange:{'id': 9, 'name': 'heading', 'supercategory': 'cube-doc'}
    27:salmon:{'id': 6, 'name': 'foot_note', 'supercategory': 'cube-doc'}
    28:pink:{'id': 7, 'name': 'footer', 'supercategory': 'cube-doc'}
    29:gold:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
    30:orchid:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
Image:
======
  id: 29
  license: 1
  file_name: 34-90503_0011_jpg.rf.f73689204e6d221e9c887abd6330c2de.jpg
  height: 2200
  width: 1700
  date_captured: 2022-03-10T03:45:06+00:00
  segmentations (8):
    184:blue:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
    185:purple:{'id': 9, 'name': 'heading', 'supercategory': 'cube-doc'}
    186:red:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
    187:green:{'id': 9, 'name': 'heading', 'supercategory': 'cube-doc'}
    188:orange:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
    189:salmon:{'id': 6, 'name': 'foot_note', 'supercategory': 'cube-doc'}
    190:pink:{'id': 7, 'name': 'footer', 'supercategory': 'cube-doc'}
    191:gold:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
Image:
======
  id: 28
  license: 1
  file_name: 34-90522_0010_jpg.rf.e7d3b8f0565b32f41134bba688e1cc5b.jpg
  height: 2200
  width: 1700
  date_captured: 2022-03-10T03:45:06+00:00
  segmentations (12):
    172:blue:{'id': 9, 'name': 'heading', 'supercategory': 'cube-doc'}
    173:purple:{'id': 3, 'name': 'bullet_enum', 'supercategory': 'cube-doc'}
    174:red:{'id': 9, 'name': 'heading', 'supercategory': 'cube-doc'}
    175:green:{'id': 3, 'name': 'bullet_enum', 'supercategory': 'cube-doc'}
    176:orange:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
    177:salmon:{'id': 6, 'name': 'foot_note', 'supercategory': 'cube-doc'}
    178:pink:{'id': 7, 'name': 'footer', 'supercategory': 'cube-doc'}
    179:gold:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
    180:orchid:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
    181:slateblue:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
    182:limegreen:{'id': 9, 'name': 'heading', 'supercategory': 'cube-doc'}
    183:seagreen:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
Image:
======
  id: 20
  license: 1
  file_name: 34-90495_0006_jpg.rf.ad90fc90fef9c5602cd068728fa89807.jpg
  height: 2200
  width: 1700
  date_captured: 2022-03-10T03:45:06+00:00
  segmentations (5):
    122:blue:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
    123:purple:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
    124:red:{'id': 7, 'name': 'footer', 'supercategory': 'cube-doc'}
    125:green:{'id': 13, 'name': 'table', 'supercategory': 'cube-doc'}
    126:orange:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
Image:
======
  id: 21
  license: 1
  file_name: 34-90544_0005_jpg.rf.026e07f906edca91cce20f666cab9f61.jpg
  height: 2200
  width: 1700
  date_captured: 2022-03-10T03:45:06+00:00
  segmentations (5):
    127:blue:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
    128:purple:{'id': 7, 'name': 'footer', 'supercategory': 'cube-doc'}
    129:red:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
    130:green:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
    131:orange:{'id': 9, 'name': 'heading', 'supercategory': 'cube-doc'}
Image:
======
  id: 5
  license: 1
  file_name: 34-90503_0009_jpg.rf.0e5e6abb509b2de9508c509a16ef5759.jpg
  height: 2200
  width: 1700
  date_captured: 2022-03-10T03:45:06+00:00
  segmentations (3):
    31:blue:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
    32:purple:{'id': 7, 'name': 'footer', 'supercategory': 'cube-doc'}
    33:red:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}

Display sample images - after annotation filtration

As seen below, the same images explored above have now only 3 classes (heading, footnote, paragraph) and hence are good to be used for training based on the assessment requirement below:

(Select only 3 classes (headings, paragraphs and footnotes) from the dataset)

In [5]:
for img_id in res:
    html = f_coco_dataset.display_image(img_id, use_url=False)
    IPython.display.display(IPython.display.HTML(html))
Image:
======
  id: 10
  license: 1
  file_name: 34-90543_0007_jpg.rf.24a847064d049989b350f9aaa88576dc.jpg
  height: 2200
  width: 1700
  date_captured: 2022-03-10T03:45:06+00:00
  segmentations (5):
    60:blue:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
    61:purple:{'id': 9, 'name': 'heading', 'supercategory': 'cube-doc'}
    62:red:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
    63:green:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
    64:orange:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
Image:
======
  id: 4
  license: 1
  file_name: 34-90552_0008_jpg.rf.87776dd4a412222469ba63b707982945.jpg
  height: 2200
  width: 1700
  date_captured: 2022-03-10T03:45:06+00:00
  segmentations (7):
    22:blue:{'id': 9, 'name': 'heading', 'supercategory': 'cube-doc'}
    23:purple:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
    24:red:{'id': 9, 'name': 'heading', 'supercategory': 'cube-doc'}
    26:green:{'id': 9, 'name': 'heading', 'supercategory': 'cube-doc'}
    27:orange:{'id': 6, 'name': 'foot_note', 'supercategory': 'cube-doc'}
    29:salmon:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
    30:pink:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
Image:
======
  id: 29
  license: 1
  file_name: 34-90503_0011_jpg.rf.f73689204e6d221e9c887abd6330c2de.jpg
  height: 2200
  width: 1700
  date_captured: 2022-03-10T03:45:06+00:00
  segmentations (7):
    184:blue:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
    185:purple:{'id': 9, 'name': 'heading', 'supercategory': 'cube-doc'}
    186:red:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
    187:green:{'id': 9, 'name': 'heading', 'supercategory': 'cube-doc'}
    188:orange:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
    189:salmon:{'id': 6, 'name': 'foot_note', 'supercategory': 'cube-doc'}
    191:pink:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
Image:
======
  id: 28
  license: 1
  file_name: 34-90522_0010_jpg.rf.e7d3b8f0565b32f41134bba688e1cc5b.jpg
  height: 2200
  width: 1700
  date_captured: 2022-03-10T03:45:06+00:00
  segmentations (9):
    172:blue:{'id': 9, 'name': 'heading', 'supercategory': 'cube-doc'}
    174:purple:{'id': 9, 'name': 'heading', 'supercategory': 'cube-doc'}
    176:red:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
    177:green:{'id': 6, 'name': 'foot_note', 'supercategory': 'cube-doc'}
    179:orange:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
    180:salmon:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
    181:pink:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
    182:gold:{'id': 9, 'name': 'heading', 'supercategory': 'cube-doc'}
    183:orchid:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
Image:
======
  id: 20
  license: 1
  file_name: 34-90495_0006_jpg.rf.ad90fc90fef9c5602cd068728fa89807.jpg
  height: 2200
  width: 1700
  date_captured: 2022-03-10T03:45:06+00:00
  segmentations (3):
    122:blue:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
    123:purple:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
    126:red:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
Image:
======
  id: 21
  license: 1
  file_name: 34-90544_0005_jpg.rf.026e07f906edca91cce20f666cab9f61.jpg
  height: 2200
  width: 1700
  date_captured: 2022-03-10T03:45:06+00:00
  segmentations (4):
    127:blue:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
    129:purple:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
    130:red:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
    131:green:{'id': 9, 'name': 'heading', 'supercategory': 'cube-doc'}
Image:
======
  id: 5
  license: 1
  file_name: 34-90503_0009_jpg.rf.0e5e6abb509b2de9508c509a16ef5759.jpg
  height: 2200
  width: 1700
  date_captured: 2022-03-10T03:45:06+00:00
  segmentations (2):
    31:blue:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}
    33:purple:{'id': 11, 'name': 'paragraph', 'supercategory': 'cube-doc'}